package com.push.spider.spider;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.hankcs.hanlp.HanLP;
import com.push.common.config.MinioConfig;
import com.push.common.config.TransConfig;
import com.push.common.manager.BeanManager;
import com.push.common.utils.*;
import com.push.common.utils.httpclient.HttpClientResult;
import com.push.common.utils.httpclient.HttpClientUtil;
import com.push.spider.utils.UrlUtils;
import com.push.web.domain.BizArticle;
import com.push.web.domain.BizArticleAppendix;
import com.push.web.domain.BizLabelSecond;
import com.push.web.domain.BizSource;
import com.push.web.service.IBizArticleAppendixService;
import com.push.web.service.IBizArticleService;
import com.push.web.service.IBizLabelSecondService;
import com.push.web.service.IBizSourceService;
import lombok.SneakyThrows;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.*;

public class TwHttpClientSpider {
    private Logger logger = LoggerFactory.getLogger(this.getClass());

    private ArrayList<String> hasDownUrls = new ArrayList<>();

    private Queue<String> queue = new LinkedList<String>();

    private BizSource source;

    private List<BizLabelSecond> labelSeconds = new ArrayList<>();

    @Autowired
    IBizArticleService articleService = BeanManager.getBean(IBizArticleService.class);

    @Autowired
    IBizSourceService sourceService = BeanManager.getBean(IBizSourceService.class);

    @Autowired
    IBizLabelSecondService labelSecondService = BeanManager.getBean(IBizLabelSecondService.class);

    @Autowired
    IBizArticleAppendixService articleAppendixService = BeanManager.getBean(IBizArticleAppendixService.class);

    @Autowired
    MinioUtil minioUtil = BeanManager.getBean(MinioUtil.class);

    @Autowired
    MinioConfig prop = BeanManager.getBean(MinioConfig.class);

    @Autowired
    private TransConfig transConfig = BeanManager.getBean(TransConfig.class);

    public int size = 0;

    public TwHttpClientSpider(String sourceId) {
        this.source = sourceService.selectBizSourceBySourceId(Long.valueOf(sourceId));
        this.labelSeconds = labelSecondService.selectBizLabelSecondListByParentIds(this.source.getLabelIds());
    }

    @SneakyThrows
    public void run(){
        queue.offer(source.getSpiderUrl());
        do{
            String url = queue.poll();
            System.out.println("url:"+url);
            if (hasDownUrls.contains(url)){
                continue;
            }
            HttpClientResult personResult = new HttpClientResult();
            if (url.startsWith("https")){
                personResult = HttpClientUtil.doGet(url, true);
            } else {
                personResult = HttpClientUtil.doGet(url);
            }
            hasDownUrls.add(url);
            size++;
            Html html = new Html(personResult.getContent());
            if (StringUtils.regex(url,source.getPageXpath())) {
                System.out.println("列表页");
                //列表页
                //System.out.println(url.toString());
                //对于从来没有检索的数据源，或者10天内没有检索过的数据源，遍历全部页码，否则，只检索前2页
                if (StringUtils.isNotEmpty(source.getPageDivXpath())){
                    List<String> links = html.xpath(source.getPageDivXpath()).links().all();
                    for (String link : links) {
                        queue.offer(UrlUtils.buildUrl(url,link));
                    }
                }
                // 如果需要爬取详情页，则继续把详情页的链接添加到爬虫队列中。
                // 如还有其他列表需要抓取，则继续添加
                //获取列表
                if (!StringUtils.isEmpty(source.getListXpath())) {
                    List<String> listXpaths = Arrays.asList(source.getListXpath().split(";"));
                    for (String listXpath:listXpaths){
                        System.out.println(html.toString());
                        List<Selectable> tagNodes = html.xpath(listXpath).nodes();
                        System.out.println(tagNodes.size());
                        if (tagNodes.size() == 0){
                            System.out.println("列表未找到节点");
                        }
                        for (Selectable contentNode : tagNodes) {
                            String titleOrg = extractInfo(source.getTitleXpath(), contentNode);
                            String sourceUrl = UrlUtils.buildUrl(url, extractInfo(source.getDetailsXpath(), contentNode));
                            String avator = UrlUtils.buildUrl(url,extractInfo(source.getAvatorXpath(), contentNode));
                            String publisher = extractInfo(source.getPublisherXpath(), contentNode);
                            if (source.getPicDownload().equalsIgnoreCase("是")){
                                avator = UrlUtils.downloadFileToMinio(avator, minioUtil);
                            }
                            String publishTime = extractInfo(source.getPublishTimeXpath(), contentNode);
                            if (StringUtils.isEmpty(sourceUrl)){
                                continue;
                            }
                            queue.offer(sourceUrl);
                            String dataKey = EncryptUtil.encrypt(sourceUrl, "md5");
                            BizArticle article = buildArticle(dataKey, titleOrg, sourceUrl, avator, publisher, publishTime, source.getSourceId());
                            BizArticle bizArticle = articleService.selectBizArticleByDataKey(dataKey);
                            if (StringUtils.isNull(bizArticle)){
                                if (strContains(article.getTitleOrg(), labelSeconds)){
                                    //自动翻译标题
                                    if (StringUtils.isContainsChinese(article.getTitleOrg())){
                                        article.setTitleCn(article.getTitleOrg());
                                    } else {
                                        String transWord = getTransResult(article.getTitleOrg(), "auto", "zh");
                                        JSONObject titleObj = JSONObject.parseObject(transWord);
                                        JSONArray titleArray = titleObj.getJSONArray("trans_result");
                                        if (StringUtils.isNotNull(titleArray) && titleArray.size() > 0){
                                            article.setTitleCn(titleArray.getJSONObject(0).getString("dst"));
                                        }
                                    }

                                    //自动翻译摘要
                                    if(transConfig.isAbstractTrans() && StringUtils.isNotEmpty(article.getSummaryOrg())){
                                        String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                                        JSONObject titleObj = JSONObject.parseObject(transWord);
                                        JSONArray titleArray = titleObj.getJSONArray("trans_result");
                                        if (StringUtils.isNotNull(titleArray) && titleArray.size() > 0){
                                            article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                                        }
                                    }

                                    articleService.insertBizArticle(article);
                                }
                            } else {
                                article.setId(bizArticle.getId());
                                articleService.updateBizArticle(article);
                            }
                        }
                    }
                }
            } else {
                System.out.println("详情页");
                // 如果有业务需要抓取详情页，则继续在这儿抓取。
                String dataKey = EncryptUtil.encrypt(url, "md5");
                //String summaryOrg = UrlUtils.repairContent(extractInfo(source.getSummaryXpath(), html), url);
                //String contentsOrg = UrlUtils.repairContent(extractInfo(source.getContentsXpath(), html), url);
                String summaryOrg = extractInfo(source.getSummaryXpath(), html);
                String contentsOrg = extractInfo(source.getContentsXpath(), html);
                if (source.getPicDownload().equalsIgnoreCase("是")){
                    List<String> imgUrls = UrlUtils.praseUrlFromString(summaryOrg, url);
                    if (imgUrls.size() > 0){
                        List<String> newImages = UrlUtils.downloadFilesToMinio(imgUrls, minioUtil);
                        summaryOrg = UrlUtils.repairDownloadContent(summaryOrg, newImages);
                    }
                    List<String> contentImgUrls = UrlUtils.praseUrlFromString(contentsOrg, url);
                    if (contentImgUrls.size() > 0){
                        List<String> newImages = UrlUtils.downloadFilesToMinio(contentImgUrls, minioUtil);
                        contentsOrg = UrlUtils.repairDownloadContent(contentsOrg, newImages);
                    }
                } else {
                    summaryOrg = UrlUtils.repairContent(summaryOrg, url);
                    contentsOrg = UrlUtils.repairContent(contentsOrg, url);
                }
                String publisher = extractInfo(source.getPublisherXpath(), html);
                String publishTime = extractInfo(source.getPublishTimeXpath(), html);
                if(StringUtils.isNotEmpty(publisher)){
                    publisher = publisher.replaceAll("来源：","");
                }
                if(StringUtils.isNotEmpty(publishTime)){
                    publishTime = publishTime.replaceAll("发布时间：","");
                }
                String avator = "";
                List<String> contentXpathList = Arrays.asList(source.getContentsXpath().split(";"));
                if(contentXpathList.size() > 0){
                    for (String contentXpath: contentXpathList){
                        List<Selectable> contentNodes = html.xpath(contentXpath).nodes();
                        if(contentNodes.size() > 0){
                            /*
                             * 单独获取图片
                             */
                            List<Selectable> imgNodes = contentNodes.get(0).xpath("//img/@src").nodes();
                            if (imgNodes.size() > 0) {
                                avator = UrlUtils.buildUrl(url, imgNodes.get(0).toString());
                                if (source.getPicDownload().equalsIgnoreCase("是")){
                                    avator = UrlUtils.downloadFileToMinio(avator, minioUtil);
                                }
                            }
                            break;
                        }
                    }
                }
                if (source.getPicDownload().equalsIgnoreCase("是")){
                    avator = UrlUtils.downloadFileToMinio(avator, minioUtil);
                }
                BizArticle article = updateArticle(dataKey, summaryOrg, contentsOrg, publisher, publishTime, avator, source.getSourceId());
                BizArticle bizArticle = articleService.selectBizArticleByDataKey(dataKey);
                if (StringUtils.isNull(bizArticle)){
                    if (strContains(article.getTitleOrg(), labelSeconds)){

                        articleService.insertBizArticle(article);
                    }
                } else {
                    article.setId(bizArticle.getId());
                    //自动翻译摘要
                    if(transConfig.isAbstractTrans() && StringUtils.isNotEmpty(article.getSummaryOrg())){
                        String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                        JSONObject titleObj = JSONObject.parseObject(transWord);
                        JSONArray titleArray = titleObj.getJSONArray("trans_result");
                        if (titleArray.size() > 0){
                            article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                        }
                    }/* else if (StringUtils.isNotEmpty(article.getContentsCn())){
                        List<String> sentenceList = HanLP.extractSummary(article.getContentsCn(),10);
                        if(sentenceList.size() > 0){
                            String abstact = sentenceList.toString();
                            article.setSummaryOrg(abstact.substring(1,abstact.lastIndexOf("]")));
                            String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                            JSONObject titleObj = JSONObject.parseObject(transWord);
                            JSONArray titleArray = titleObj.getJSONArray("trans_result");
                            if (titleArray.size() > 0){
                                article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                            }
                        }
                    }*/
                    articleService.updateBizArticle(article);
                }
                if(StringUtils.isNotEmpty(source.getFilesXpath())){
                    List<String> filesXpathList = Arrays.asList(source.getFilesXpath().split(";"));
                    List<BizArticleAppendix> appendices = new ArrayList<>();
                    for (String fileXpath: filesXpathList){
                        List<Selectable> fileNodes = html.xpath(fileXpath).nodes();
                        if(fileNodes.size() > 0){
                            for (Selectable fileNode:fileNodes){
                                BizArticleAppendix appendix = new BizArticleAppendix();
                                appendix.setAppendixUrl(fileNode.toString());
                                appendix.setArticleId(article.getId());
                            }
                        }
                    }
                    if (!appendices.isEmpty()){
                        articleAppendixService.batchSave(appendices);
                    }
                }
            }
            Thread.sleep(source.getIntervalNum()*1000);
        } while (queue.size() > 0);
    }

    public String extractInfo(String xpathString, Selectable contentNode){
        if (StringUtils.isEmpty(xpathString)){
            return "";
        }
        List<String> xpathList = Arrays.asList(xpathString.split(";"));
        for (String xpath: xpathList){
            List<Selectable> nodes = contentNode.xpath(xpath).nodes();
            if(nodes.size() > 0){
                return nodes.get(0).toString();
            }
        }
        return "";
    }

    public BizArticle buildArticle(String dataKey, String titleOrg, String url,String avator, String publisher, String publishTime, long sourceId){
        BizArticle article = new BizArticle();
        article.setDataKey(dataKey);
        article.setTitleOrg(titleOrg);
        article.setUrl(url);
        article.setAvator(avator);
        article.setPublisher(publisher);
        article.setPublishTime(publishTime);
        article.setPublishFormatTime(DateUtils.formatTwDate(publishTime));
        article.setSourceId(sourceId);
        article.setCreateTime(DateUtils.getNowDate());
        return article;
    }

    public BizArticle updateArticle(String dataKey, String summaryOrg, String contentsOrg, String publisher, String publishTime, String avator, long sourceId){
        BizArticle article = new BizArticle();
        article.setDataKey(dataKey);
        article.setSummaryOrg(summaryOrg);
        contentsOrg.replaceAll("visibility: hidden;", "visibility: visible;");
        article.setContentsOrg(contentsOrg);
        article.setContentsCn(StringUtils.delHtmlTags(contentsOrg));
        article.setPublisher(publisher);
        article.setPublishTime(publishTime);
        article.setPublishFormatTime(DateUtils.formatTwDate(publishTime));
        article.setAvator(avator);
        article.setSourceId(sourceId);
        article.setCreateTime(DateUtils.getNowDate());
        return article;
    }

    public static boolean strContains(String str, List<BizLabelSecond> labels){
        if (labels == null){
            return true;
        }
        if (labels.isEmpty()){
            return true;
        }
        if(StringUtils.isEmpty(str)){
            return false;
        }
        long len = labels.parallelStream().filter(label -> str.toLowerCase(Locale.ROOT).contains(label.getLabelName().toLowerCase(Locale.ROOT))).count();
        if (len > 0){
            return true;
        }
        return false;
    }

    public String getTransResult(String query, String from, String to) {
        Map<String, String> params = buildParams(query, from, to);
        return HttpClientUtil.doGet(transConfig.getTransUrl(), params,false).getContent();
    }

    private Map<String, String> buildParams(String query, String from, String to) {
        Map<String, String> params = new HashMap<String, String>();
        params.put("q", query);
        params.put("from", from);
        params.put("to", to);
        params.put("appid", transConfig.getAppid());
        // 随机数
        String salt = String.valueOf(System.currentTimeMillis());
        params.put("salt", salt);
        // 签名
        String src = transConfig.getAppid() + query + salt + transConfig.getSecurityKey();
        params.put("sign", MD5.md5(src));
        return params;
    }

    public int getSize(){
        return size;
    }
}
